# encoding=utf-8
import sys
sys.path.append("../mysql/")
sys.path.append("../mongo/")
sys.path.append("../")
import mysql_helper
import mongo_helper
import mylogger
import jieba

punctuation = r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~“”？，！【】（）、。：；’‘……￥· """
PAGE_SIZE = 50
index = {}
stop_word = []
start_id = 0

def gen_inverted_index(data):
    '''
    :param data: [[docid, text], [docid, text], [docid, text], ... ]
    :return:
    '''
    global index
    for item in data:
        seg_list = jieba.cut(item[1])
        for word in seg_list:
            if word in stop_word or len(word) == 0:
                continue
            if index.get(word) is None:
                index[word] = [[item[0], 1]]
            else:
                word_index = index.get(word)
                k = 0
                for docid, count in word_index:
                    if docid == item[0]:
                        index[word][k][1] = count + 1
                        break
                    k +=1
                if k == len(word_index):
                    index[word].append([item[0], 1])

def read_mysql_gen_index():
    global index
    global stop_word
    global start_id
    doc_num = mongo_helper.get_doc_num()
    with open("continue_id", "r") as file:
        start_id = int(file.readline())
    with open("stop_word", "r", encoding="utf-8") as file2:
        stop_word = file2.readlines()
        stop_word = [word.replace("\n", "") for word in stop_word]

    while True:
        data = mysql_helper.get_mblog_by_start_id(start_id, PAGE_SIZE)
        if len(data) == 0:
            print("异常终止")
            break
        gen_inverted_index(data)
        start_id = data[-1][0] + 1
        doc_num += len(data)

        if len(index) >= 40000:
            write_index_to_mongodb(index)
            with open("continue_id", "w") as file:
                file.write(str(start_id))
            mongo_helper.save_doc_num(doc_num)
            index = {}

def write_index_to_mongodb(index):
    for key, value in index.items():
        doc = mongo_helper.find_by_item(key)
        if doc is None:
            mongo_helper.insert_into_index(key, value)
        else:
            #需要把doc和value融合
            doc = doc.get("index")
            doc = doc + value
            mongo_helper.update_by_item(key, doc)

if __name__ == "__main__":
    # with open("stop_word", "r", encoding="utf-8") as file2:
    #     stop_word = file2.readlines()
    #     stop_word = [word.replace("\n", "") for word in stop_word]
    # data = [[0, "【致敬！#生活中那些让人破防的拼搏瞬间#】在我们看得见、看不见的角落里，有无数努力拼搏的人，无论他们身在乡村还是城市，无论从事何种职业，都在拼尽全力地奋斗着。正因如此，才让这个世界变得更加美好。致敬每一个奋斗者！  http://t.cn/A6Ms2BN0 ​​​"]]
    # gen_inverted_index(data)
    read_mysql_gen_index()
    # print(index)

